{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "SAC的简化版实现,alpha使用常量代替.只使用一个value模型,而不是两个."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAT/ElEQVR4nO3df2xT57kH8K+dxIaQHIeQxV6UWCCtG8340S1AOOt0V60uWRdNZUTaD6EuqxAI5qDSbFw1EqWit1I6tnvZutGgaRrwT8eUSbSXXEoVhRLuVpdAWK6SANE6USVrsV1gPk5SYif2c/9AOashsDgJfu34+5GOhM/72H7OSfzlnPfYsUVEBEREClhVN0BE2YsBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyigLoIMHD2Lp0qVYsGABqqur0dXVpaoVIlJESQD94Q9/QGNjI1588UVcvHgRq1evRk1NDYLBoIp2iEgRi4oPo1ZXV2Pt2rX49a9/DQCIx+OoqKjAzp078fzzz6e6HSJSJDfVTxiNRtHd3Y2mpiZzndVqhcfjgc/nm/I+kUgEkUjEvB2Px3Hz5k0sWbIEFovlgfdMRMkREQwPD6OsrAxW671PtFIeQNevX0csFoPT6UxY73Q6ceXKlSnv09zcjH379qWiPSKaQ0NDQygvL7/neMoDaCaamprQ2Nho3jYMA263G0NDQ9A0TWFnRDSVcDiMiooKFBYW3rcu5QFUUlKCnJwcBAKBhPWBQAAul2vK+9jtdtjt9rvWa5rGACJKY/9qiiTlV8FsNhuqqqrQ0dFhrovH4+jo6ICu66luh4gUUnIK1tjYiPr6eqxZswbr1q3DL37xC4yOjuKZZ55R0Q4RKaIkgL773e/i448/xt69e+H3+/HII4/g1KlTd01ME9H8puR9QLMVDofhcDhgGAbngIjS0HRfo/wsGBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRJOoDOnj2Lb33rWygrK4PFYsEbb7yRMC4i2Lt3Lz772c9i4cKF8Hg8+Otf/5pQc/PmTWzevBmapqGoqAhbtmzByMjIrDaEiDJP0gE0OjqK1atX4+DBg1OO79+/H6+++ioOHTqEc+fOYdGiRaipqcHY2JhZs3nzZvT396O9vR1tbW04e/Ystm3bNvOtIKLMJLMAQI4fP27ejsfj4nK55Gc/+5m5LhQKid1ul9///vciInLp0iUBIOfPnzdr3nrrLbFYLPLhhx9O63kNwxAAYhjGbNonogdkuq/ROZ0Dunr1Kvx+Pzwej7nO4XCguroaPp8PAODz+VBUVIQ1a9aYNR6PB1arFefOnZvycSORCMLhcMJCRJlvTgPI7/cDAJxOZ8J6p9Npjvn9fpSWliaM5+bmori42Ky5U3NzMxwOh7lUVFTMZdtEpEhGXAVramqCYRjmMjQ0pLolIpoDcxpALpcLABAIBBLWBwIBc8zlciEYDCaMT0xM4ObNm2bNnex2OzRNS1iIKPPNaQAtW7YMLpcLHR0d5rpwOIxz585B13UAgK7rCIVC6O7uNmtOnz6NeDyO6urquWyHiNJcbrJ3GBkZwfvvv2/evnr1Knp6elBcXAy3241du3bh5ZdfxkMPPYRly5bhhRdeQFlZGTZu3AgAePjhh/GNb3wDW7duxaFDhzA+Po6GhgZ873vfQ1lZ2ZxtGBFlgGQvr73zzjsC4K6lvr5eRG5fin/hhRfE6XSK3W6Xxx9/XAYGBhIe48aNG/L9739fCgoKRNM0eeaZZ2R4eHjOL/ERkRrTfY1aREQU5t+MhMNhOBwOGIbB+SCiNDTd12hGXAUjovmJAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpk/TX8hA9CLHoLYQG+wCJ315hscBRsQK59ny1jdEDxQCitBAd/QeunjkCiY0DACzWHFRu2sMAmud4CkZpQeJx3P6KOcomDCBKCzJ56kVZhQFEaUHicR4AZSEGEKUHianugBRgAFFauD0HRNmGAURp4fYcEM/Bsk1SAdTc3Iy1a9eisLAQpaWl2LhxIwYGBhJqxsbG4PV6sWTJEhQUFKCurg6BQCChZnBwELW1tcjPz0dpaSl2796NiYmJ2W8NZSweAWWnpAKos7MTXq8X7733Htrb2zE+Po4NGzZgdHTUrHnuuedw4sQJtLa2orOzEx999BE2bdpkjsdiMdTW1iIajeLdd9/F0aNHceTIEezdu3futooyjnAOKDvJLASDQQEgnZ2dIiISCoUkLy9PWltbzZrLly8LAPH5fCIicvLkSbFareL3+82alpYW0TRNIpHItJ7XMAwBIIZhzKZ9SiP/+OD/5PxvtkvXoa3SdWirnP/Ndhm9/nfVbdEMTfc1Oqs5IMMwAADFxcUAgO7uboyPj8Pj8Zg1y5cvh9vths/nAwD4fD6sXLkSTqfTrKmpqUE4HEZ/f/+UzxOJRBAOhxMWml9E4hDhHFC2mXEAxeNx7Nq1C48++ihWrFgBAPD7/bDZbCgqKkqodTqd8Pv9Zs2nw2dyfHJsKs3NzXA4HOZSUVEx07YpTUnsjjlAiwWwqOmFUmfGAeT1etHX14djx47NZT9TampqgmEY5jI0NPTAn5NS61bI/88PogLIy3cgd0GBwo4oFWb0YdSGhga0tbXh7NmzKC8vN9e7XC5Eo1GEQqGEo6BAIACXy2XWdHV1JTze5FWyyZo72e122O32mbRKGULiiZPQFmsOLBa+S2S+S+onLCJoaGjA8ePHcfr0aSxbtixhvKqqCnl5eejo6DDXDQwMYHBwELquAwB0XUdvby+CwaBZ097eDk3TUFlZOZttoXnEYrHCYuE52HyX1BGQ1+vF66+/jjfffBOFhYXmnI3D4cDChQvhcDiwZcsWNDY2ori4GJqmYefOndB1HevXrwcAbNiwAZWVlXj66aexf/9++P1+7NmzB16vl0c59CkWgEdA815SAdTS0gIAeOyxxxLWHz58GD/84Q8BAAcOHIDVakVdXR0ikQhqamrw2muvmbU5OTloa2vDjh07oOs6Fi1ahPr6erz00kuz2xKaVywWy+2JaJrXLJKB1z7D4TAcDgcMw4CmaarboTnw9/Nv4trF/zFvLywux8NP/TtybAsUdkUzNd3XKI9xKS3xCCg7MIAoPTGAsgIDiNKSxWKFhe9EnPcYQJSeeASUFRhAlJb4PqDswAAi5aa8EMsjoKzAAKL0cFcIWcBPo85/DCBKC/xanuzEAKK0wD/Jmp0YQJQeeASUlRhAlAbkrj/HQdmBAURpgXNA2YkBROoJAM4BZSUGEKUFHgFlJwYQpQHOAWUrBhClBR4BZScGEKUFvg8oOzGASDkRQOKJ3wvGD6JmBwYQqScxjIUCCasWLC5T1AylEgOIlBORu46ArLk2Rd1QKjGAKC1ZrPzVzAb8KVNa4reiZgf+lCktWaw5qlugFGAAUVriKVh2SOqbUYlmanR0FOPj41OOxSeiiN/xPqCxSBSGYdzz8QoLC5GTw6OkTMcAopT4yU9+ghMnTkw5Zs/LwYFtX4Vr8SJz3c9//p/473M7p6y32Ww4deoUPv/5zz+QXil1GECUEjdv3sSHH3445Zg9LwejURv6R7+KT2Ia3Asu4cY/fPest9ls9zyaosyS1Il2S0sLVq1aBU3ToGkadF3HW2+9ZY6PjY3B6/ViyZIlKCgoQF1dHQKBxDeYDQ4Oora2Fvn5+SgtLcXu3bsxMTFx51NRFhHkoG/k3zA09jBujJejd+QxfBxxqW6LUiCpACovL8crr7yC7u5uXLhwAV//+tfx1FNPob+/HwDw3HPP4cSJE2htbUVnZyc++ugjbNq0ybx/LBZDbW0totEo3n33XRw9ehRHjhzB3r1753arKKMILBiJFWHyWzAmxIaR8XylPVGKyCwtXrxYfvvb30ooFJK8vDxpbW01xy5fviwAxOfziYjIyZMnxWq1it/vN2taWlpE0zSJRCLTfk7DMASAGIYx2/YpRb7zne8Ibv/psbsWe16e/PT5n8p/vOyTfS93yX81n5LHq9fes95ms0lfX5/qTaL7mO5rdMZzQLFYDK2trRgdHYWu6+ju7sb4+Dg8Ho9Zs3z5crjdbvh8Pqxfvx4+nw8rV66E0+k0a2pqarBjxw709/fjS1/6UlI9XLlyBQUFBTPdBEqhcDh8z7GJ2AR8fz6Ksdw/41a8AKW2QXww9P4960UEf/vb3/iB1TQ2MjIyrbqkA6i3txe6rmNsbAwFBQU4fvw4Kisr0dPTA5vNhqKiooR6p9MJv98PAPD7/QnhMzk+OXYvkUgEkUjEvD35y2wYBuePMkQ0Gr3nWCwueON/LwG4NO3HGx4eRigUmn1j9ECMjo5Oqy7pAPrCF76Anp4eGIaBP/7xj6ivr0dnZ2fSDSajubkZ+/btu2t9dXU1NE17oM9Nc6OkpGTOHstiseCRRx7BF7/4xTl7TJpb9zvi/bSk325qs9nwuc99DlVVVWhubsbq1avxy1/+Ei6XC9Fo9K7/lQKBAFyu21c0XC7XXVfFJm9P1kylqakJhmGYy9DQULJtE1EamvX73ePxOCKRCKqqqpCXl4eOjg5zbGBgAIODg9B1HQCg6zp6e3sRDAbNmvb2dmiahsrKyns+h91uNy/9Ty5ElPmSOgVramrCk08+CbfbjeHhYbz++us4c+YM3n77bTgcDmzZsgWNjY0oLi6GpmnYuXMndF3H+vXrAQAbNmxAZWUlnn76aezfvx9+vx979uyB1+uF3W5/IBtIROkrqQAKBoP4wQ9+gGvXrsHhcGDVqlV4++238cQTTwAADhw4AKvVirq6OkQiEdTU1OC1114z75+Tk4O2tjbs2LEDuq5j0aJFqK+vx0svvTS3W0VpZ+HChXN25Gqz2WDlh1XnBYuIiOomkhUOh+FwOGAYBk/HMsT169dx69atOXs8l8uFvLy8OXs8mlvTfY3ys2CUEnN5FYzmDx7HEpEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImVyVTcwEyICAAiHw4o7IaKpTL42J1+r95KRAXTjxg0AQEVFheJOiOh+hoeH4XA47jmekQFUXFwMABgcHLzvxlGicDiMiooKDA0NQdM01e1kBO6zmRERDA8Po6ys7L51GRlAVuvtqSuHw8FfihnQNI37LUncZ8mbzsEBJ6GJSBkGEBEpk5EBZLfb8eKLL8Jut6tuJaNwvyWP++zBssi/uk5GRPSAZOQREBHNDwwgIlKGAUREyjCAiEiZjAyggwcPYunSpViwYAGqq6vR1dWluiVlmpubsXbtWhQWFqK0tBQbN27EwMBAQs3Y2Bi8Xi+WLFmCgoIC1NXVIRAIJNQMDg6itrYW+fn5KC0txe7duzExMZHKTVHmlVdegcViwa5du8x13GcpIhnm2LFjYrPZ5He/+5309/fL1q1bpaioSAKBgOrWlKipqZHDhw9LX1+f9PT0yDe/+U1xu90yMjJi1mzfvl0qKiqko6NDLly4IOvXr5evfOUr5vjExISsWLFCPB6P/OUvf5GTJ09KSUmJNDU1qdiklOrq6pKlS5fKqlWr5NlnnzXXc5+lRsYF0Lp168Tr9Zq3Y7GYlJWVSXNzs8Ku0kcwGBQA0tnZKSIioVBI8vLypLW11ay5fPmyABCfzyciIidPnhSr1Sp+v9+saWlpEU3TJBKJpHYDUmh4eFgeeughaW9vl6997WtmAHGfpU5GnYJFo1F0d3fD4/GY66xWKzweD3w+n8LO0odhGAD++YHd7u5ujI+PJ+yz5cuXw+12m/vM5/Nh5cqVcDqdZk1NTQ3C4TD6+/tT2H1qeb1e1NbWJuwbgPsslTLqw6jXr19HLBZL+KEDgNPpxJUrVxR1lT7i8Th27dqFRx99FCtWrAAA+P1+2Gw2FBUVJdQ6nU74/X6zZqp9Ojk2Hx07dgwXL17E+fPn7xrjPkudjAoguj+v14u+vj786U9/Ut1KWhsaGsKzzz6L9vZ2LFiwQHU7WS2jTsFKSkqQk5Nz19WIQCAAl8ulqKv00NDQgLa2NrzzzjsoLy8317tcLkSjUYRCoYT6T+8zl8s15T6dHJtvuru7EQwG8eUvfxm5ubnIzc1FZ2cnXn31VeTm5sLpdHKfpUhGBZDNZkNVVRU6OjrMdfF4HB0dHdB1XWFn6ogIGhoacPz4cZw+fRrLli1LGK+qqkJeXl7CPhsYGMDg4KC5z3RdR29vL4LBoFnT3t4OTdNQWVmZmg1Joccffxy9vb3o6ekxlzVr1mDz5s3mv7nPUkT1LHiyjh07Jna7XY4cOSKXLl2Sbdu2SVFRUcLViGyyY8cOcTgccubMGbl27Zq5fPLJJ2bN9u3bxe12y+nTp+XChQui67roum6OT15S3rBhg/T09MipU6fkM5/5TFZdUv70VTAR7rNUybgAEhH51a9+JW63W2w2m6xbt07ee+891S0pA2DK5fDhw2bNrVu35Ec/+pEsXrxY8vPz5dvf/rZcu3Yt4XE++OADefLJJ2XhwoVSUlIiP/7xj2V8fDzFW6POnQHEfZYa/HMcRKRMRs0BEdH8wgAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiU+X/QWKL3BeHCfwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('CartPole-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, terminated, truncated, info = self.env.step(action)\n",
    "        over = terminated or truncated\n",
    "\n",
    "        #限制最大步数\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            over = True\n",
    "\n",
    "        return state, reward, over\n",
    "\n",
    "    #打印游戏图像\n",
    "    def show(self):\n",
    "        from matplotlib import pyplot as plt\n",
    "        plt.figure(figsize=(3, 3))\n",
    "        plt.imshow(self.env.render())\n",
    "        plt.show()\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()\n",
    "\n",
    "env.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.5751, 0.4249],\n",
       "        [0.5584, 0.4416]], grad_fn=<SoftmaxBackward0>)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "model_action = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 2),\n",
    "    torch.nn.Softmax(dim=1),\n",
    ")\n",
    "\n",
    "model_action(torch.randn(2, 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0334, -0.1868],\n",
       "        [-0.0077, -0.0947]], grad_fn=<AddmmBackward0>)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_value = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 2),\n",
    ")\n",
    "\n",
    "model_value_next = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 2),\n",
    ")\n",
    "\n",
    "model_value_next.load_state_dict(model_value.state_dict())\n",
    "\n",
    "model_value(torch.randn(2, 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\appDir\\python3.10\\lib\\site-packages\\gym\\utils\\passive_env_checker.py:233: DeprecationWarning: `np.bool8` is a deprecated alias for `np.bool_`.  (Deprecated NumPy 1.24)\n",
      "  if not isinstance(terminated, (bool, np.bool8)):\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "11.0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "#玩一局游戏并记录数据\n",
    "def play(show=False):\n",
    "    data = []\n",
    "    reward_sum = 0\n",
    "\n",
    "    state = env.reset()\n",
    "    over = False\n",
    "    while not over:\n",
    "        prob = model_action(torch.FloatTensor(state).reshape(1, 4))[0].tolist()\n",
    "        action = random.choices(range(2), weights=prob, k=1)[0]\n",
    "\n",
    "        next_state, reward, over = env.step(action)\n",
    "\n",
    "        data.append((state, action, reward, next_state, over))\n",
    "        reward_sum += reward\n",
    "\n",
    "        state = next_state\n",
    "\n",
    "        if show:\n",
    "            display.clear_output(wait=True)\n",
    "            env.show()\n",
    "\n",
    "    return data, reward_sum\n",
    "\n",
    "\n",
    "play()[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Administrator\\AppData\\Local\\Temp\\ipykernel_15052\\3891364554.py:27: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ..\\torch\\csrc\\utils\\tensor_new.cpp:248.)\n",
      "  state = torch.FloatTensor([i[0] for i in data]).reshape(-1, 4)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(211,\n",
       " (array([-0.03882636,  0.01716037, -0.04174506, -0.04671043], dtype=float32),\n",
       "  0,\n",
       "  1.0,\n",
       "  array([-0.03848315, -0.1773389 , -0.04267927,  0.23251481], dtype=float32),\n",
       "  False))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#数据池\n",
    "class Pool:\n",
    "\n",
    "    def __init__(self):\n",
    "        self.pool = []\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.pool)\n",
    "\n",
    "    def __getitem__(self, i):\n",
    "        return self.pool[i]\n",
    "\n",
    "    #更新动作池\n",
    "    def update(self):\n",
    "        #每次更新不少于N条新数据\n",
    "        old_len = len(self.pool)\n",
    "        while len(pool) - old_len < 200:\n",
    "            self.pool.extend(play()[0])\n",
    "\n",
    "        #只保留最新的N条数据\n",
    "        self.pool = self.pool[-2_0000:]\n",
    "\n",
    "    #获取一批数据样本\n",
    "    def sample(self):\n",
    "        data = random.sample(self.pool, 64)\n",
    "\n",
    "        state = torch.FloatTensor([i[0] for i in data]).reshape(-1, 4)\n",
    "        action = torch.LongTensor([i[1] for i in data]).reshape(-1, 1)\n",
    "        reward = torch.FloatTensor([i[2] for i in data]).reshape(-1, 1)\n",
    "        next_state = torch.FloatTensor([i[3] for i in data]).reshape(-1, 4)\n",
    "        over = torch.LongTensor([i[4] for i in data]).reshape(-1, 1)\n",
    "\n",
    "        return state, action, reward, next_state, over\n",
    "\n",
    "\n",
    "pool = Pool()\n",
    "pool.update()\n",
    "state, action, reward, next_state, over = pool.sample()\n",
    "\n",
    "len(pool), pool[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer_action = torch.optim.Adam(model_action.parameters(), lr=2e-4)\n",
    "optimizer_value = torch.optim.Adam(model_value.parameters(), lr=2e-3)\n",
    "\n",
    "\n",
    "def soft_update(_from, _to):\n",
    "    for _from, _to in zip(_from.parameters(), _to.parameters()):\n",
    "        value = _to.data * 0.995 + _from.data * 0.005\n",
    "        _to.data.copy_(value)\n",
    "\n",
    "\n",
    "def get_prob_entropy(state):\n",
    "    prob = model_action(torch.FloatTensor(state).reshape(-1, 4))\n",
    "    entropy = prob * (prob + 1e-8).log()\n",
    "    entropy = -entropy.sum(dim=1, keepdim=True)\n",
    "\n",
    "    return prob, entropy\n",
    "\n",
    "\n",
    "def requires_grad(model, value):\n",
    "    for param in model.parameters():\n",
    "        param.requires_grad_(value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0103167295455933"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def train_value(state, action, reward, next_state, over):\n",
    "    requires_grad(model_value, True)\n",
    "    requires_grad(model_action, False)\n",
    "\n",
    "    #计算target\n",
    "    with torch.no_grad():\n",
    "        #计算动作的熵\n",
    "        prob, entropy = get_prob_entropy(next_state)\n",
    "        target = model_value_next(next_state)\n",
    "\n",
    "    #加权熵,熵越大越好\n",
    "    target = (prob * target).sum(dim=1, keepdim=True)\n",
    "    target = target + 1e-3 * entropy\n",
    "    target = target * 0.98 * (1 - over) + reward\n",
    "\n",
    "    #计算value\n",
    "    value = model_value(state).gather(dim=1, index=action)\n",
    "    loss = torch.nn.functional.mse_loss(value, target)\n",
    "    loss.backward()\n",
    "    optimizer_value.step()\n",
    "    optimizer_value.zero_grad()\n",
    "\n",
    "    return loss.item()\n",
    "\n",
    "\n",
    "train_value(state, action, reward, next_state, over)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.033202797174453735"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def train_action(state):\n",
    "    requires_grad(model_value, False)\n",
    "    requires_grad(model_action, True)\n",
    "\n",
    "    #计算熵\n",
    "    prob, entropy = get_prob_entropy(state)\n",
    "\n",
    "    #计算value\n",
    "    value = model_value(state)\n",
    "\n",
    "    #求期望求和\n",
    "    value = (prob * value).sum(dim=1, keepdim=True)\n",
    "\n",
    "    #加权熵\n",
    "    loss = -(value + 1e-3 * entropy).mean()\n",
    "\n",
    "    loss.backward()\n",
    "    optimizer_action.step()\n",
    "    optimizer_action.zero_grad()\n",
    "\n",
    "    return loss.item()\n",
    "\n",
    "\n",
    "train_action(state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "OHoSU6uI-xIt",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 432 23.85\n",
      "10 2478 17.15\n",
      "20 4567 69.2\n",
      "30 7068 198.45\n",
      "40 9686 183.85\n"
     ]
    }
   ],
   "source": [
    "def train():\n",
    "    model_action.train()\n",
    "    model_value.train()\n",
    "\n",
    "    #训练N次\n",
    "    for epoch in range(50):\n",
    "        #更新N条数据\n",
    "        pool.update()\n",
    "\n",
    "        #每次更新过数据后,学习N次\n",
    "        for i in range(50):\n",
    "            #采样一批数据\n",
    "            state, action, reward, next_state, over = pool.sample()\n",
    "\n",
    "            #训练\n",
    "            train_value(state, action, reward, next_state, over)\n",
    "            train_action(state)\n",
    "            soft_update(model_value, model_value_next)\n",
    "\n",
    "        if epoch % 10 == 0:\n",
    "            test_result = sum([play()[-1] for _ in range(20)]) / 20\n",
    "            print(epoch, len(pool), test_result)\n",
    "\n",
    "\n",
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAVrUlEQVR4nO3dbWxc1Z0G8OeO58Wvd/wWz+DaJmlhCVZeSp3EmSJt2eLGpRZLwIvaKktdFIFIxxHB3Ui1BEFhqYxSaWlpg/MBNeHD0iC3SrtYAWTZwahiyItTS45JvFAF7CaZGZzgGdvJvP/3A+sbhjiOx3bmeMzzk67EPefMzP8euA+ec+/MaCIiICJSwKS6ACL66mIAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMsoCaO/evVi+fDmys7NRW1uLY8eOqSqFiBRREkCvv/46Wlpa8Oyzz+LkyZNYu3Yt6uvr4ff7VZRDRIpoKj6MWltbi/Xr1+N3v/sdACCRSKCyshLbt2/HL37xi3SXQ0SKmNP9gpFIBH19fWhtbTXaTCYT6urq4PF4pn1MOBxGOBw29hOJBC5duoSSkhJomnbTayai1IgIxsfHUV5eDpPp+m+00h5Ao6OjiMfjcDgcSe0OhwNnzpyZ9jFtbW3YvXt3OsojogU0MjKCioqK6/anPYDmorW1FS0tLcZ+IBBAVVUVRkZGoOu6wsqIaDrBYBCVlZUoKCiYcVzaA6i0tBRZWVnw+XxJ7T6fD06nc9rH2Gw22Gy2a9p1XWcAES1iN1oiSftVMKvVipqaGnR3dxttiUQC3d3dcLlc6S6HiBRS8haspaUFTU1NWLduHTZs2IBf//rXmJycxKOPPqqiHCJSREkA/fCHP8Snn36KXbt2wev14pvf/CbeeuutaxamiWhpU3If0HwFg0HY7XYEAgGuAREtQrM9R/lZMCJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESmTcgC9++67uP/++1FeXg5N0/DnP/85qV9EsGvXLtxyyy3IyclBXV0dPvzww6Qxly5dwpYtW6DrOgoLC7F161ZMTEzM60CIKPOkHECTk5NYu3Yt9u7dO23/nj178NJLL2Hfvn04evQo8vLyUF9fj1AoZIzZsmULBgcH0dXVhc7OTrz77rt4/PHH534URJSZZB4AyKFDh4z9RCIhTqdTfvWrXxltY2NjYrPZ5A9/+IOIiHzwwQcCQI4fP26MefPNN0XTNDl37tysXjcQCAgACQQC8ymfiG6S2Z6jC7oGdPbsWXi9XtTV1RltdrsdtbW18Hg8AACPx4PCwkKsW7fOGFNXVweTyYSjR49O+7zhcBjBYDBpI6LMt6AB5PV6AQAOhyOp3eFwGH1erxdlZWVJ/WazGcXFxcaYL2tra4Pdbje2ysrKhSybiBTJiKtgra2tCAQCxjYyMqK6JCJaAAsaQE6nEwDg8/mS2n0+n9HndDrh9/uT+mOxGC5dumSM+TKbzQZd15M2Isp8CxpAK1asgNPpRHd3t9EWDAZx9OhRuFwuAIDL5cLY2Bj6+vqMMT09PUgkEqitrV3IcohokTOn+oCJiQl89NFHxv7Zs2fR39+P4uJiVFVVYceOHXj++edx++23Y8WKFXjmmWdQXl6OzZs3AwDuvPNOfP/738djjz2Gffv2IRqNorm5GT/60Y9QXl6+YAdGRBkg1ctrR44cEQDXbE1NTSLy+aX4Z555RhwOh9hsNrn33ntlaGgo6TkuXrwoP/7xjyU/P190XZdHH31UxsfHF/wSHxGpMdtzVBMRUZh/cxIMBmG32xEIBLgeRLQIzfYczYirYES0NDGAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkTMq/C/ZVIpJA7MoEwhMXcXl0GNa8ItirVkPTNNWlES0JDKAZBD4ZwCfvvY7o5QAkHoVeUQ29ohpaFqeNaCHwLdgMsmy5iF0JQuJRAEAo4EMiFlFcFdHSwQCaQXahAyazzdiPXRlHKOBTWBHR0sIAmkGWJRs5JV8z9hOxCEJjDCCihcIAmoGWZUFOUXlSW/Afg8jAX7MmWpQYQDPQNA361+4EcPWqV2RyDBKPqSuKaAlJKYDa2tqwfv16FBQUoKysDJs3b8bQ0FDSmFAoBLfbjZKSEuTn56OxsRE+X/LbluHhYTQ0NCA3NxdlZWXYuXMnYrHFeVJb8wqTrnpdvjiCaGhcYUVES0dKAdTb2wu32433338fXV1diEaj2LRpEyYnJ40xTz31FN544w10dHSgt7cX58+fx0MPPWT0x+NxNDQ0IBKJ4L333sOrr76KAwcOYNeuXQt3VAsou9ABm15q7CdiUUTGLyqsiGgJkXnw+/0CQHp7e0VEZGxsTCwWi3R0dBhjTp8+LQDE4/GIiMjhw4fFZDKJ1+s1xrS3t4uu6xIOh2f1uoFAQABIIBCYT/mzkojH5Mwb/yXH9j1mbBf635ZEInHTX5soU832HJ3XGlAgEAAAFBcXAwD6+voQjUZRV1dnjFm5ciWqqqrg8XgAAB6PB6tXr4bD4TDG1NfXIxgMYnBwcNrXCYfDCAaDSVvaaCboFXcmNY17/w5wIZpo3uYcQIlEAjt27MDdd9+NVatWAQC8Xi+sVisKCwuTxjocDni9XmPMF8Nnqn+qbzptbW2w2+3GVllZOdeyU6ZpGnJLKpPWgcIBHySxONesiDLJnAPI7Xbj1KlTOHjw4ELWM63W1lYEAgFjGxkZuemv+UXZhU6YsizGfmTiEkLBT9NaA9FSNKcAam5uRmdnJ44cOYKKigqj3el0IhKJYGxsLGm8z+eD0+k0xnz5qtjU/tSYL7PZbNB1PWlLJ3N2HrKLbjH249EQopOfpbUGoqUopQASETQ3N+PQoUPo6enBihUrkvprampgsVjQ3d1ttA0NDWF4eBgulwsA4HK5MDAwAL/fb4zp6uqCruuorq6ez7HcNCazDbb8kqS24Pn/5Q2JRPOU0se63W43XnvtNfzlL39BQUGBsWZjt9uRk5MDu92OrVu3oqWlBcXFxdB1Hdu3b4fL5cLGjRsBAJs2bUJ1dTUeeeQR7NmzB16vF08//TTcbjdsNttML6+MpmkoKL8Dl/5+3GgLfXYBkojzk/FE85DS2dPe3g4AuOeee5La9+/fj5/+9KcAgBdffBEmkwmNjY0Ih8Oor6/Hyy+/bIzNyspCZ2cntm3bBpfLhby8PDQ1NeG5556b35HcZDlFt0AzmY3F5wn/WcQjV2DKKVBcGVHm0iQD30cEg0HY7XYEAoG0rQfFQhMY/NMvEZn4/CZEkyUbK//1P5BXWpWW1yfKJLM9R/lZsFkyWWyw5hcZ+4loCJc//YTrQETzwACaJc1kRr7ztqS2K5+dV1QN0dLAAJolTdOQbS8DvvB90BO+s4AkFFZFlNkYQCnId3wj6YbEUMCHMD+YSjRnDKAUWHLtsOnLjP145Api/GoOojljAKUgy5oDa97VhWhIAsFzZ9QVRJThGEApKii/I2k/PH4RwnUgojlhAKUo216WtD/h/QiJKH+qh2guGEAp0DQNuaVVMGdfvfs5FppELDyhsCqizMUASpHZlossa46xHwtP4PLoP3hDItEcMIBSZLLYUFD+T1cbRDA5+om6gogyGAMoZRpyir+W1DLpP6uoFqLMxgBKkaZpyHd8HdCuTl14fBSxENeBiFLFAJoDa64dlpyrn/CNTHyG6BXekEiUKgbQHFhy7bAWFBv7kojj8qdcByJKFQNoLjQT8kpvvbovCVy+xCthRKliAM1R3rJbk/bHz53hb8YTpYgBNAeapiGvbPmX7geaRDxyRWFVRJmHATRH5hwdWdZsYz8y+Rm/oIwoRQygOTJbc5BX9vWrDSL8sUKiFDGA5kgzZSV9RzQAjJ8f4kI0UQoYQPNgr1iVtH/l0jmuAxGlgAE0Dza9BObsfGM/MnER8fBlhRURZRYG0DxY84qT7ohOxKKYHOVP9RDNFn9X+AZEBBMTE4jH49P0JaDllwH/f/VLEnEE/SMwFX/jus+naRoKCgpgMjH7iRhANxCPx/Hwww/j1KlT0/bfv+FWPLqpGsDnP9fT/T//jd0Pb0fiOn8FLVu2DD09PSgqKpq2n+irhAF0AyICv9+Pc+fOTdt/dCCOB/65FsPRWsTFDHNWLy6N+jEZmv5rWmOxGBIJfoc0EZDiGlB7ezvWrFkDXdeh6zpcLhfefPNNoz8UCsHtdqOkpAT5+flobGyEz+dLeo7h4WE0NDQgNzcXZWVl2LlzJ2KxzP0IwyejUZwY+xd4I1/Hp9Fb4bc9iGXL+HvxRLORUgBVVFTghRdeQF9fH06cOIHvfve7eOCBBzA4OAgAeOqpp/DGG2+go6MDvb29OH/+PB566CHj8fF4HA0NDYhEInjvvffw6quv4sCBA9i1a9fCHlUaxcSMy/ECTL0FiyEPJUWlaosiyhQyT0VFRfLKK6/I2NiYWCwW6ejoMPpOnz4tAMTj8YiIyOHDh8VkMonX6zXGtLe3i67rEg6HZ/2agUBAAEggEJhv+TcUiUTkrrvuEgDTbmazTf59y/Py3PNHZfd/HpUXf/knaf63e6473uFwyOjo6E2vm0il2Z6jc14Disfj6OjowOTkJFwuF/r6+hCNRlFXV2eMWblyJaqqquDxeLBx40Z4PB6sXr0aDofDGFNfX49t27ZhcHAQd911V0o1nDlzBvn5+TceOA+xWAyhUGiG/jCGTr4CZ44Xg598hs+8x/HhJyPXHR+PxzE0NITCwsKbUC3R4jAxMbtvCE05gAYGBuByuRAKhZCfn49Dhw6huroa/f39sFqt15xYDocDXq8XAOD1epPCZ6p/qu96wuEwwuGwsR8MBgEAgUDgpq8fxePxaS/Bf9GJMx/j5NBexBM3vv8nkUgY9RMtVZOTk7Mal3IA3XHHHejv70cgEMAf//hHNDU1obe3N+UCU9HW1obdu3df015bWwtd16d5xMKJRqPIy8ubcYwIEJ/lzYcWiwXr169HSUnJQpRHtCjN9n+yKd8NZ7Vacdttt6GmpgZtbW1Yu3YtfvOb38DpdCISiWBsbCxpvM/ng9PpBAA4nc5rropN7U+NmU5raysCgYCxjYxc/y0OEWWOed+Om0gkEA6HUVNTA4vFgu7ubqNvaGgIw8PDcLlcAACXy4WBgQH4/X5jTFdXF3RdR3V19XVfw2azGZf+pzYiynwpvQVrbW3Ffffdh6qqKoyPj+O1117DO++8g7fffht2ux1bt25FS0sLiouLoes6tm/fDpfLhY0bNwIANm3ahOrqajzyyCPYs2cPvF4vnn76abjdbthstptygES0eKUUQH6/Hz/5yU9w4cIF2O12rFmzBm+//Ta+973vAQBefPFFmEwmNDY2IhwOo76+Hi+//LLx+KysLHR2dmLbtm1wuVzIy8tDU1MTnnvuuYU9qgWWl5e3YH916boOTdMW5LmIMp0mknkf3Q4Gg7Db7QgEAjf97ZiIwOfzIRqNLsjzmUwmOJ1OZGVlLcjzES1Gsz1H+VmwG9A0bcYFciKaO34nBBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlLGrLqAuRARAEAwGFRcCRFNZ+rcnDpXrycjA+jixYsAgMrKSsWVENFMxsfHYbfbr9ufkQFUXFwMABgeHp7x4ChZMBhEZWUlRkZGoOu66nIyAudsbkQE4+PjKC8vn3FcRgaQyfT50pXdbud/FHOg6zrnLUWcs9TN5o8DLkITkTIMICJSJiMDyGaz4dlnn4XNZlNdSkbhvKWOc3ZzaXKj62RERDdJRv4FRERLAwOIiJRhABGRMgwgIlImIwNo7969WL58ObKzs1FbW4tjx46pLkmZtrY2rF+/HgUFBSgrK8PmzZsxNDSUNCYUCsHtdqOkpAT5+flobGyEz+dLGjM8PIyGhgbk5uairKwMO3fuRCwWS+ehKPPCCy9A0zTs2LHDaOOcpYlkmIMHD4rVapXf//73Mjg4KI899pgUFhaKz+dTXZoS9fX1sn//fjl16pT09/fLD37wA6mqqpKJiQljzBNPPCGVlZXS3d0tJ06ckI0bN8q3v/1toz8Wi8mqVaukrq5O/va3v8nhw4eltLRUWltbVRxSWh07dkyWL18ua9askSeffNJo55ylR8YF0IYNG8Ttdhv78XhcysvLpa2tTWFVi4ff7xcA0tvbKyIiY2NjYrFYpKOjwxhz+vRpASAej0dERA4fPiwmk0m8Xq8xpr29XXRdl3A4nN4DSKPx8XG5/fbbpaurS77zne8YAcQ5S5+MegsWiUTQ19eHuro6o81kMqGurg4ej0dhZYtHIBAAcPUDu319fYhGo0lztnLlSlRVVRlz5vF4sHr1ajgcDmNMfX09gsEgBgcH01h9erndbjQ0NCTNDcA5S6eM+jDq6Ogo4vF40r90AHA4HDhz5oyiqhaPRCKBHTt24O6778aqVasAAF6vF1arFYWFhUljHQ4HvF6vMWa6OZ3qW4oOHjyIkydP4vjx49f0cc7SJ6MCiGbmdrtx6tQp/PWvf1VdyqI2MjKCJ598El1dXcjOzlZdzldaRr0FKy0tRVZW1jVXI3w+H5xOp6KqFofm5mZ0dnbiyJEjqKioMNqdTicikQjGxsaSxn9xzpxO57RzOtW31PT19cHv9+Nb3/oWzGYzzGYzent78dJLL8FsNsPhcHDO0iSjAshqtaKmpgbd3d1GWyKRQHd3N1wul8LK1BERNDc349ChQ+jp6cGKFSuS+mtqamCxWJLmbGhoCMPDw8acuVwuDAwMwO/3G2O6urqg6zqqq6vTcyBpdO+992JgYAD9/f3Gtm7dOmzZssX4Z85ZmqheBU/VwYMHxWazyYEDB+SDDz6Qxx9/XAoLC5OuRnyVbNu2Tex2u7zzzjty4cIFY7t8+bIx5oknnpCqqirp6emREydOiMvlEpfLZfRPXVLetGmT9Pf3y1tvvSXLli37Sl1S/uJVMBHOWbpkXACJiPz2t7+VqqoqsVqtsmHDBnn//fdVl6QMgGm3/fv3G2OuXLkiP/vZz6SoqEhyc3PlwQcflAsXLiQ9z8cffyz33Xef5OTkSGlpqfz85z+XaDSa5qNR58sBxDlLD34dBxEpk1FrQES0tDCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZf4PK7NfBAstFxQAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "174.0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "play(True)[-1]"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "第7章-DQN算法.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
